package zunge.lbe1;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;

import zunge.tools.DbConn;


public class ClusteringFolds {
	
	List<DataFold> dataFolds;
	int K = 20; //Max(#Clusters)
	int N = 90; //#(Random starting points for kmeans)
	int C = 3; //#Cores used for each kmeans calculation process
	int CORE = 5; //#Processes
	
	int F = 5; //#(Top discriminative features of each cluster)
	
	public ClusteringFolds(List<DataFold> dataFolds){
		this.dataFolds = dataFolds;
	}
	
	/** label: center, label, size, log, report, heatmap*/
	public String getMFile(int method, int v, int k, String tmpDir, String label){
		return tmpDir + "cluster_m"+method+"_v"+v+"_k"+k+"_"+label+".txt";
	}
	/** svmMethod=1,2,3,4,5,6;  label=train,test,model,out,log */
	public String getMFileSVM(int method, int svmMethod, int v, int k, String tmpDir, String label){
		return tmpDir + "cluster_m"+method+"_v"+v+"_k"+k+"_svmm"+svmMethod+"_"+label+".txt";
	}
	
	
	
	
	/**Generate Script for K-Means Clustering
	 * Run KMeans on all data folds, compare the membership of users.
	 */
	public void gen(ScriptGen sg) throws Exception{
		List<String> scripts = new ArrayList<String>();
		
		for(DataFold dataFold:dataFolds){
			genM(4, dataFold, scripts, sg);
			//genM(5, dataFold, scripts, sg);
			//genM(6, dataFold, scripts, sg);
		}
		sg.splitScript(scripts, CORE, "cluster", "cluster.sh",
				dataFolds.get(0).data.getScriptDir(), dataFolds.get(0).data.getTempDir());
	}
	
	void genM(int method, DataFold dataFold, List<String> scripts, ScriptGen sg){
		
		String fin_uf = Flow3.getMFile(method, dataFold.v, dataFold.data.getTempDir(), "uf");

		//Scan through all possible Ks
		for(int k = 1;k <= K;++k){
			String fout_center = getMFile(method, dataFold.v, k, dataFold.data.getTempDir(), "center");
			String fout_label = getMFile(method, dataFold.v, k, dataFold.data.getTempDir(), "label");
			String fout_size = getMFile(method, dataFold.v, k, dataFold.data.getTempDir(), "size");
			String fout_log = getMFile(method, dataFold.v, k, dataFold.data.getTempDir(), "log");
			
			scripts.add(sg.genRKmeans(k, method, C, fin_uf, fout_center, fout_label, fout_size, fout_log));
		}
	}
	
	
	
	
	
	
	
	/** Collect the k-means result, 
	 * 		for each k
	 * 			calculate stability among different folds, 
	 * 			for each fold
	 * 				generate reports (discriminative items)
	 * 				generate i/o and script for classification		
	 * */
	public void collectAndGen(ScriptGen sg, DbConn conn) throws Exception{
		List<String> scripts = new ArrayList<String>();
		
		collectAndGenM(4, sg, conn, scripts);
		//collectAndGenM(5, sg, conn, scripts);
		//collectAndGenM(6, sg, conn, scripts);
		
		sg.splitScript(scripts, Data.CORES, "clusterTrain", "clusterTrain.sh",
				dataFolds.get(0).data.getScriptDir(), dataFolds.get(0).data.getTempDir());
	}
	
	void collectAndGenM(int method, ScriptGen sg, DbConn conn, List<String> scripts) throws Exception{
		
		for(int k = 2; k <= K; ++k){
		
			DataFold dataFold = dataFolds.get(0);
			
			//for(DataFold dataFold:dataFolds){
				System.out.println("collectAndGen k="+k+" v="+dataFold.v);
				
				
				//Read Cluster labels for each user
				int uMapSize = dataFold.data.uMap.size();
				int[] clabels = new int[uMapSize];
				String f_label = getMFile(method, dataFold.v, k, dataFold.data.getTempDir(), "label");
				BufferedReader br = new BufferedReader(new FileReader(f_label));
				for(int i = 0;i < uMapSize;++i){
					clabels[i] = Integer.parseInt(br.readLine());
				}
				br.close();
				System.out.println("\tclabels["+clabels.length+"]");
				
				switch(method){
				case 4:
					//Get labels for all permissions
					//String[] mlabels = dataFold.data.getPLabels(conn);
					//String f_report = getMFile(method, dataFold.v, k, dataFold.data.getTempDir(), "report");
					//generateReportM4(dataFold, f_report, k, clabels, mlabels, conn);
					
					//Generate HeatMap for the data
					//String f_heatmap = getMFile(method, dataFold.v, k, dataFold.data.getTempDir(), "heatmap");
					//generateHeatMapM4(dataFold, k, Data.P, clabels, f_heatmap);
					
					//Generate Script for classification
					generateSVM(dataFold, k, clabels, sg, scripts);
				}
				
			//}
		}
		
	}
	
	
	
	
	
	static final String tableName_cupdm4 = "tmp_cupdm4";
	void generateReportM4(DataFold dataFold, String freport, int k, int[] clabels, String[] mlabels, DbConn conn) throws Exception{
		System.out.println("generateReportM4 v="+dataFold.v+" k="+k);
		
		//Generate Temporary Table on SQL Database
		String query = "drop table if exists "+tableName_cupdm4;
		conn.executeUpdate(conn.getPreparedStatement(query));
		query = "create table "+tableName_cupdm4+" (c int, u int, p int, d double, primary key (u,p))";
		conn.executeUpdate(conn.getPreparedStatement(query));
		
		//Read matrix and store into table
		System.out.println("\treadmatrix");
		String f_m4matrix = Flow3.getMFile(4, dataFold.v, dataFold.data.getTempDir(), "matrix");
		BufferedReader br = new BufferedReader(new FileReader(f_m4matrix));
		int uid,pid;double d;
		PreparedStatement ps = conn.getPreparedStatement("insert into "+tableName_cupdm4+" (c,u,p,d) values (?,?,?,?)");
		String line;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[1]);
			d = Double.parseDouble(frags[2]);
			if(uid > dataFold.data.uMap.size()){continue;}
			ps.setInt(1, clabels[uid-1]-1);
			ps.setInt(2, uid-1);
			ps.setInt(3, pid-1);
			ps.setDouble(4, d);
			conn.executeUpdate(ps);
		}
		br.close();
		
		/* Search for top-confident discriminative patterns for each cluster */
		//Generate patterns
		System.out.println("\tgenerate patterns");
		List<M4PatternSet> patternSets = new ArrayList<M4PatternSet>();
		//Generate 1-dim patterns
		for(int m = 0;m < 12;++m){
			patternSets.add(new M4PatternSet(conn, k, 1, m, -1));
		}
		
		//Generate 2-dim patterns
		for(int m1 = 0;m1 < 11;++m1){
			for(int m2 = m1+1; m2 < 12;++m2){
				patternSets.add(new M4PatternSet(conn, k, 2, m1, m2));
			}
		}
		
		//Write Report
		System.out.println("\twrite report");
		BufferedWriter bw = new BufferedWriter(new FileWriter(freport));
		bw.write("Report for K="+k+" M="+12+"\n");
		double totalTopC = 0;
		for(int c = 0;c < k;++c){
			
			//Generate top-F patterns by F-measure
			List<M4Pattern> patterns = new ArrayList<M4Pattern>();
			for(M4PatternSet patternSet:patternSets){
				patterns.add(patternSet.getPattern(c, M4Pattern.TYPE_PURITYEDGE));
			}
			Collections.sort(patterns);
			
			bw.write("Cluster "+c+" size="+patterns.get(0).patternSet.cSize[c]+"\n");
			
			totalTopC += patterns.get(0).confidenceValue;
			for(int i = 0;i < F && i < patterns.size();++i){
				M4Pattern pattern = patterns.get(i);
				bw.write("\tTop "+(i+1)+":\t");
				switch(pattern.d){
				case 1:
					bw.write(getDecision(pattern.d1)+"("+mlabels[pattern.m1]+") "
							+" [C="+getDecimal(pattern.confidenceValue)+"] ");
							//+pattern.precision+" "+pattern.recall+" "+pattern.purityEdge+"\n");
					break;
				case 2:
					bw.write(getDecision(pattern.d1)+"("+mlabels[pattern.m1]+") "
							+getDecision(pattern.d2)+"("+mlabels[pattern.m2]+") "
							+" [C="+getDecimal(pattern.confidenceValue)+"] ");
							//+pattern.precision+" "+pattern.recall+" "+pattern.purityEdge+"\n");
					break;
				}
				bw.write("\n");
			}
		}
		bw.write("avg top confidence value = "+totalTopC/k+"\n");
		bw.close();
		
	}
	String getDecision(int d){
		if(d==1){return "Accept";}
		else if(d==-1){return "Reject";}
		else{return null;}
	}
	String getDecimal(double d){
		return String.format("%.2f", d);
	}
	
	
	class M4PatternSet{
		int D; //depth of pattern search
		int S; //power(2,d)
		int m1;
		int m2;
		
		double[][] cCount;
		double[] totalCount;
		double[] cSize;
		
		double[][] precision; //given cluster, ratio of users satisfying settings.
		double[][] recall; //given settings, ratio of users who fall into this cluster.
		double[][] fmeasure;
		double[][] purityEdge; //considering the percentage of users who satisfy settings, what is the edge of this cluster comparing to average value?
		
		void init(int k){
			cCount = new double[k][S]; for(int i =0;i<k;++i){Arrays.fill(cCount[i], 0);}
			totalCount = new double[S]; Arrays.fill(totalCount, 0);
			cSize = new double[k];Arrays.fill(cSize, 0);
			
			precision = new double[k][S];for(int i =0;i<k;++i){Arrays.fill(precision[i], 0);}
			recall = new double[k][S];for(int i =0;i<k;++i){Arrays.fill(recall[i], 0);}
			fmeasure = new double[k][S];for(int i =0;i<k;++i){Arrays.fill(fmeasure[i], 0);}
			purityEdge = new double[k][S];for(int i =0;i<k;++i){Arrays.fill(purityEdge[i], 0);}
		}
		
		/** d = depth of search, d=1,2.
		 */
		M4PatternSet(DbConn conn, int k, int D, int m1, int m2) throws Exception{
			this.D = D;
			this.S = (int)Math.pow(2, D);
			this.m1 = m1;
			this.m2 = m2;
			init(k);
			
			//Get Original Data
			ResultSet rs;
			for(int c = 0;c < k;++c){
				rs = conn.executeQuery("select count(distinct u) from "+tableName_cupdm4+" where c = "+c);
				rs.next();
				cSize[c] = rs.getDouble(1);
				rs.close();
			}
			
			switch(D){
			case 1:
				for(int s = 0;s < 2;++s){//encoding of settings: 0:-, 1:+
					String criteria = (s==0)?"d < 0":"d >= 0";
					
					totalCount[s] = getUserSet(m1, criteria, -1, conn).size();
					for(int c = 0;c < k;++c){
						cCount[c][s] = getUserSet(m1, criteria, c, conn).size();
					}
				}
				break;
			case 2:
				for(int s = 0;s < 4;++s){//encoding of settings: 0:--, 1:-+, 2:+-, 3:++
					String criteria1 = (s<2)?"d < 0":"d >= 0";
					String criteria2 = (s%2==0)?"d < 0":"d >= 0";
					
					HashSet<Integer> userSet = getUserSet(m1, criteria1, -1, conn);
					//System.err.println("set1.size="+userSet.size());
					userSet.retainAll(getUserSet(m2, criteria2, -1, conn));
					totalCount[s] = userSet.size();
					if(totalCount[s]==0){System.err.println("totalCount=0 D="+D+" s="+s);}
					
					for(int c = 0;c < k;++c){
						userSet = getUserSet(m1, criteria1, c, conn);
						userSet.retainAll(getUserSet(m2, criteria2, c, conn));
						cCount[c][s] = userSet.size();
					}
				}
				break;
			}
			
			
			//Calc stat values
			for(int s = 0;s < S;++s){
				for(int c = 0;c < k;++c){
					if(cSize[c]>0){precision[c][s] = cCount[c][s] / cSize[c];}
					//else{System.out.println("cSize["+c+"]="+cSize[c]);}
					
					if(totalCount[s]>0){recall[c][s] = cCount[c][s] / totalCount[s];}
					//else{System.out.println("totalCount["+s+"]="+totalCount[s]+" recall["+c+"]["+s+"]="+recall[c][s]);}
					
					if(precision[c][s] + recall[c][s]>0){
						fmeasure[c][s] = 2*precision[c][s]*recall[c][s] / (precision[c][s] + recall[c][s]);
					}
					
					for(int c2 = 0;c2 < k;++c2){
						purityEdge[c][s] += (precision[c][s] - precision[c2][s]);
					}
					if(K>2){purityEdge[c][s] /= (k-1);}
				}
			}
		}
		/** c=-1 for general statistics */
		HashSet<Integer> getUserSet(int p, String criteria, int c, DbConn conn) throws Exception{
			String query = (c==-1)
					?"select distinct u from "+tableName_cupdm4+" where p = "+p+" and "+criteria
					:"select distinct u from "+tableName_cupdm4+" where p = "+p+" and c = "+c+" and "+criteria;
			HashSet<Integer> userSet = new HashSet<Integer>();
			ResultSet rs = conn.executeQuery(query);
			while(rs.next()){
				userSet.add(rs.getInt(1));
			}
			rs.close();
			//System.out.println("getUserSet p="+p+" criteria="+criteria+" c="+c+" resultSize="+userSet.size());
			return userSet;
		}
		
		
		/** Generate pattern for each cluster on these features (pick the feature settings with max confidence value) */
		M4Pattern getPattern(int c, int type) throws Exception{
			double[][] measure;
			switch(type){
			case M4Pattern.TYPE_FMEASURE:
				measure = fmeasure;break;
			case M4Pattern.TYPE_PURITYEDGE:
				measure = purityEdge;break;
			default:
				throw new Exception("wrong type");
			}
			
				
			int maxS = -1;
			double maxV = -10000;
			for(int s = 0;s < S;++s){if(measure[c][s]>maxV){maxS = s;maxV = measure[c][s];}}
			
			switch(D){
			case 1://encoding of settings: 0:-, 1:+
				switch(maxS){
				case 0: return new M4Pattern(this, type, c, D, m1, -1, -1, -1, measure[c][0], precision[c][0],recall[c][0],purityEdge[c][0]);
				case 1: return new M4Pattern(this, type, c, D, m1, 1, -1, -1, measure[c][1], precision[c][1],recall[c][1],purityEdge[c][1]);
				default: throw new Exception("wrong maxS");
				}
			case 2://encoding of settings: 0:--, 1:-+, 2:+-, 3:++
				switch(maxS){
				case 0: return new M4Pattern(this, type, c, D, m1, -1, m2, -1, measure[c][0], precision[c][0],recall[c][0],purityEdge[c][0]);
				case 1: return new M4Pattern(this, type, c, D, m1, -1, m2, 1, measure[c][1], precision[c][1],recall[c][1],purityEdge[c][1]);
				case 2: return new M4Pattern(this, type, c, D, m1, 1, m2, -1, measure[c][2], precision[c][2],recall[c][2],purityEdge[c][2]);
				case 3: return new M4Pattern(this, type, c, D, m1, 1, m2, 1, measure[c][3], precision[c][3],recall[c][3],purityEdge[c][3]);
				default: throw new Exception("wrong maxS");
				}
			default:
				throw new Exception("wrong D");
			}
		}
	}

	
	class M4Pattern implements Comparable<M4Pattern>{
		public M4PatternSet patternSet;
		public int type;
		public static final int TYPE_FMEASURE = 0;
		public static final int TYPE_PURITYEDGE = 1;
		
		public Double confidenceValue;
		public int c; //clusterId
		public int d; //depth of search
		public int m1,m2;
		public int d1,d2;
		
		public double precision;
		public double recall;
		public double purityEdge;
		
		public M4Pattern(M4PatternSet patternSet, int type, int c, int d, int m1, int d1, int m2, int d2, double cfv, 
				double precision, double recall, double purityEdge){
			this.patternSet = patternSet;
			this.type = type;
			this.c = c;
			this.d = d;
			this.m1 = m1; this.d1 = d1;
			this.m2 = m2; this.d2 = d2;
			this.confidenceValue = cfv;
			this.precision = precision;
			this.recall = recall;
			this.purityEdge = purityEdge;
		}

		@Override
		public int compareTo(M4Pattern arg0) {
			return arg0.confidenceValue.compareTo(this.confidenceValue);
		}
	}
	
	
	
	void generateHeatMapM4(DataFold dataFold, int k, int m, int[] clabels, String f_heatmap) throws Exception{
		System.out.println("generateHeatMap");
		
		double[][] prec = new double[k][m]; for(int i = 0;i < k;++i){Arrays.fill(prec[i], 0);}
		int[] cSize = new int[k]; Arrays.fill(cSize, 0);
		
		String f_m4matrix = Flow3.getMFile(4, dataFold.v, dataFold.data.getTempDir(), "matrix");
		BufferedReader br = new BufferedReader(new FileReader(f_m4matrix));
		int uid,pid;double d;
		String line;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[1]);
			d = Double.parseDouble(frags[2]);
			if(uid > dataFold.data.uMap.size()){continue;}
			
			cSize[clabels[uid-1]-1] ++;
			prec[clabels[uid-1]-1][pid-1] += d;
		}
		br.close();
		for(int c = 0;c < k;++c){
			if(cSize[c]>0){
				for(int p = 0;p < m;++p){
					prec[c][p] /= cSize[c];
				}
			}
		}
		
		BufferedWriter bw = new BufferedWriter(new FileWriter(f_heatmap));
		for(int c = 0;c < k;++c){
			for(int p = 0;p < m;++p){
				bw.write(prec[c][p]+" ");
			}
			bw.write("\n");
		}
		bw.close();
	}
	
	
	//Here we only train on Method 3 and 4
	void generateSVM(DataFold dataFold, int k, int[] clabels, ScriptGen sg, List<String> scripts) throws Exception{
		
		String m3train = Flow2.getMFile(dataFold.v, dataFold.data.getTempDir(), 3, "train");
		String m3test = Flow2.getMFile(dataFold.v, dataFold.data.getTempDir(), 3, "test");
		String svm4m3train = getMFileSVM(4, 3, dataFold.v, k, dataFold.data.getTempDir(), "train");
		String svm4m3test = getMFileSVM(4, 3, dataFold.v, k, dataFold.data.getTempDir(), "test");
		String svm4m3model = getMFileSVM(4, 3, dataFold.v, k, dataFold.data.getTempDir(), "model");
		String svm4m3out = getMFileSVM(4, 3, dataFold.v, k, dataFold.data.getTempDir(), "out");
		String svm4m3log = getMFileSVM(4, 3, dataFold.v, k, dataFold.data.getTempDir(), "log");
		
		translatem3(m3train, svm4m3train, k, clabels, dataFold.data);
		translatem3(m3test, svm4m3test, k, clabels, dataFold.data);
		new File(svm4m3model).delete();
		new File(svm4m3out).delete();
		new File(svm4m3log).delete();
		
		scripts.add(sg.genLibLinear(svm4m3train, svm4m3test, svm4m3model, svm4m3out, svm4m3log));
		
		
		String m4train = Flow3.getMFile(4, dataFold.v, dataFold.data.getTempDir(), "lltrain");
		String m4test = Flow3.getMFile(4, dataFold.v, dataFold.data.getTempDir(), "lltest");
		String svm4m4train = getMFileSVM(4, 4, dataFold.v, k, dataFold.data.getTempDir(), "train");
		String svm4m4test = getMFileSVM(4, 4, dataFold.v, k, dataFold.data.getTempDir(), "test");
		String svm4m4model = getMFileSVM(4, 4, dataFold.v, k, dataFold.data.getTempDir(), "model");
		String svm4m4out = getMFileSVM(4, 4, dataFold.v, k, dataFold.data.getTempDir(), "out");
		String svm4m4log = getMFileSVM(4, 4, dataFold.v, k, dataFold.data.getTempDir(), "log");
		
		translatem4(m4train, svm4m4train, k, clabels, dataFold.data);
		translatem4(m4test, svm4m4test, k, clabels, dataFold.data);
		new File(svm4m4model).delete();
		new File(svm4m4out).delete();
		new File(svm4m4log).delete();
		
		scripts.add(sg.genLibLinear(svm4m4train, svm4m4test, svm4m4model, svm4m4out, svm4m4log));
		
	}	
	void translatem3(String fin, String fout, int k, int[] clabels, Data data) throws Exception{
		BufferedReader br = new BufferedReader(new FileReader(fin));
		BufferedWriter bw = new BufferedWriter(new FileWriter(fout));
		String line;
		int uid,cid;
		while((line=br.readLine())!=null){
			String uidStr = line.substring(line.lastIndexOf(" ")+1);
			uid = Integer.parseInt(uidStr.substring(0, uidStr.indexOf(":")));
			if(uid > clabels.length){continue;}
			cid = clabels[uid-1];
			bw.write(line.substring(0,line.lastIndexOf(" ")+1)+
					(cid+Data.P+data.aMap.size())    +":1\n");
		}	
		br.close();
		bw.close();
	}
	void translatem4(String fin, String fout, int k, int[] clabels, Data data) throws Exception{
		BufferedReader br = new BufferedReader(new FileReader(fin));
		BufferedWriter bw = new BufferedWriter(new FileWriter(fout));
		String line;
		int uid,cid;double featureValue;
		while((line=br.readLine())!=null){
			String trivialStr = line.substring(0, line.lastIndexOf(" "));
			String uidStr = trivialStr.substring(trivialStr.lastIndexOf(" ")+1);
			uid = Integer.parseInt(uidStr.substring(0, uidStr.indexOf(":")));
			if(uid > clabels.length){continue;}
			cid = clabels[uid-1];
			
			String featureStr = line.substring(line.lastIndexOf(" ")+1);
			featureValue = Double.parseDouble(featureStr.substring(featureStr.indexOf(":")+1));
			
			bw.write(trivialStr.substring(0, trivialStr.lastIndexOf(" ")+1)+
					(cid+Data.P+data.aMap.size())+":1 "+(data.aMap.size() + Data.P + k + 1)+":"+featureValue+"\n");
		}

		br.close();
		bw.close();
	}
	
	void collectResult() throws Exception{
		collectResultM(3);
		//collectResultM(4);
	}
	
	void collectResultM(int svmMethod) throws Exception{		
		for(int k = 2; k <= K; ++k){
			String resultFile = dataFolds.get(0).data.getResultDir()+"clusterm4_svmm"+svmMethod+"_k"+k+".txt";
			BufferedWriter bw = new BufferedWriter(new FileWriter(resultFile));
			
			double totalAcc = 0;
			
			DataFold dataFold = dataFolds.get(0);
			
			//for(DataFold dataFold:dataFolds){
				System.out.println("collectAndGen k="+k+" v="+dataFold.v);
				
				String svmlog = getMFileSVM(4, svmMethod, dataFold.v, k, dataFold.data.getTempDir(), "log");
				BufferedReader br = new BufferedReader(new FileReader(svmlog));
				String line = br.readLine();
				double acc = Double.parseDouble(line.substring(line.indexOf("=")+2, line.indexOf("%")));
				totalAcc += acc;
				bw.write(acc+" ");
				br.close();
			//}
			bw.write("\n"+totalAcc+"\n");
			bw.close();
		}
	}
	
}
